{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%run NB01-Load.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import gc\n",
    "import IPython\n",
    "import lightgbm as lgb\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import plotly.graph_objs as go\n",
    "import plotly.plotly as py\n",
    "import random \n",
    "#import seaborn as sb\n",
    "import seaborn as sns\n",
    "#import seaborn as snss\n",
    "#import scipy\n",
    "import scipy as sp\n",
    "import sklearn\n",
    "import statsmodels.api as sm\n",
    "import statsmodels.formula.api as smf\n",
    "import sys\n",
    "import warnings\n",
    "\n",
    "from datetime import date\n",
    "from IPython.display import HTML\n",
    "from lightgbm import LGBMClassifier\n",
    "from lightgbm import plot_importance\n",
    "from pandas import DataFrame\n",
    "from pandas import Series\n",
    "from patsy import dmatrices\n",
    "from plotly import tools\n",
    "from plotly.offline import init_notebook_mode\n",
    "from plotly.offline import iplot\n",
    "from pylab import rcParams\n",
    "from random import choice\n",
    "from random import choices # Python 3.6+\n",
    "from random import sample\n",
    "from sklearn import datasets\n",
    "from sklearn import model_selection\n",
    "from sklearn import metrics\n",
    "from sklearn import preprocessing\n",
    "#from sklearn.cross_validation import train_test_split\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import Imputer\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from statsmodels.stats.outliers_influence import variance_inflation_factor\n",
    "from xgboost import XGBClassifier\n",
    "from wordcloud import WordCloud\n",
    "\n",
    "init_notebook_mode(connected=True)\n",
    "plt.rcParams.update({'figure.max_open_warning': 200})\n",
    "# Suppress warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "# In a notebook environment, display the plots inline\n",
    "%matplotlib inline\n",
    "# Set some parameters to apply to all plots. These can be overridden in each plot if desired\n",
    "# Plot size to 14\" x 7\"\n",
    "matplotlib.rc('figure', figsize = (14, 7))\n",
    "# Font size to 14\n",
    "matplotlib.rc('font', size = 14)\n",
    "# Do not display top and right frame lines\n",
    "matplotlib.rc('axes.spines', top = False, right = False)\n",
    "# Remove grid lines\n",
    "matplotlib.rc('axes', grid = False)\n",
    "# Set backgound color to white\n",
    "matplotlib.rc('axes', facecolor = 'white')\n",
    "\n",
    "np.set_printoptions(threshold=np.nan)\n",
    "np.set_printoptions(suppress=True, formatter={'float_kind':'{:f}'.format})\n",
    "\n",
    "#import C:/Development/kaggle--home-credit-default-risk/rand_jitter\n",
    "#import C:/Development/kaggle--home-credit-default-risk/draw_feature_distribution\n",
    "import sys\n",
    "# sys.path.insert(0, 'C:/Development/kaggle--home-credit-default-risk/') # ~= sys.path.prepend\n",
    "sys.path.append('C:/Development/kaggle--home-credit-default-risk/')\n",
    "# import rand_jitter\n",
    "# import draw_feature_distribution\n",
    "##from rand_jitter import * # NOTE: added directly to draw_feature_distribution_v2\n",
    "# from draw_feature_distribution import *\n",
    "# from draw_feature_distribution_v1 import *\n",
    "from draw_feature_distribution_v2 import *\n",
    "\n",
    "# C:\\Users\\jbalcomb\\Anaconda3\\lib\\site-packages\\statsmodels\\compat\\pandas.py:56: FutureWarning:\n",
    "# The pandas.core.datetools module is deprecated and will be removed in a future version. Please use the pandas.tseries module instead.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Python version: {}\".format(sys.version))\n",
    "print(\"pandas version: {}\".format(pd.__version__))\n",
    "print(\"NumPy version: {}\".format(np.__version__))\n",
    "print(\"SciPy version: {}\".format(sp.__version__))\n",
    "print(\"scikit-learn version: {}\".format(sklearn.__version__))\n",
    "print(\"matplotlib version: {}\".format(matplotlib.__version__))\n",
    "print(\"IPython version: {}\".format(IPython.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(1234567890)\n",
    "numpy.random.seed(1234567890)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_file_path = 'C:/Development/kaggle--home-credit-default-risk/data/'\n",
    "data_file_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#application_train = pd.read_table(path + 'application_train.csv', sep=',', dtype=object)\n",
    "application_train = pd.read_table(data_file_path + 'application_train.csv', sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#application_train[other_columns] = ds13[other_columns].fillna(value = \"MISSING\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train__ext_source = application_train.loc[:, ['EXT_SOURCE_1','EXT_SOURCE_2','EXT_SOURCE_3','TARGET']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove observations with missing values\n",
    "#application_train__ext_source.dropna(inplace = True)\n",
    "application_train__ext_source__dropna = application_train__ext_source.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create train and validation set\n",
    "X_train, X_test, y_train, y_test = train_test_split(application_train__ext_source, application_train__ext_source['TARGET'], random_state=0)\n",
    "#train_x, valid_x, train_y, valid_y = train_test_split(data, y, test_size=0.2, shuffle=True, stratify=y, random_state=1301)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"X_train shape: {}\".format(X_train.shape))\n",
    "print(\"y_train shape: {}\".format(y_train.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"X_test shape: {}\".format(X_test.shape))\n",
    "print(\"y_test shape: {}\".format(y_test.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train model\n",
    "model = logreg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predict class\n",
    "model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# View predicted probabilities\n",
    "model.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logisticRegr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logisticRegr.fit(ds_train, convert_train.values.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(\"Score=%.3f\" % clf.score(X, grades[\"Letter\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(logisticRegr.score(ds_test, convert_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = logisticRegr.predict(ds_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(logisticRegr.score(ds_test, convert_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cm = confusion_matrix(predictions, convert_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(pd.DataFrame(cm, columns=labels, index=labels))\n",
    "#print(pd.DataFrame(cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 1234567890"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold = model_selection.KFold(n_splits=10, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LogisticRegression(class_weight='balanced')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scoring = 'accuracy'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = model_selection.cross_val_score(model, X_train, y_train, cv=kfold, scoring=scoring)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Accuracy: %.3f (%.3f)\") % (results.mean(), results.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(results.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(results.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logit = sm.Logit(y_train, X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = logit.fit(maxiter=999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print result.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#------------------------Build LightGBM Model-----------------------\n",
    "train_data=lgb.Dataset(train_x,label=train_y)\n",
    "valid_data=lgb.Dataset(valid_x,label=valid_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://medium.com/@sunwoopark/kaggle-%EB%8F%84%EC%A0%84%EA%B8%B0-home-credit-default-risk-part-1-735030d40ee0\n",
    "### SMOTE\n",
    "from imblearn.over_sampling import SMOTE\n",
    "X_resampled, y_resampled = SMOTE().fit_sample(X_train, y_train)\n",
    "\n",
    "### ROC_AUC_SCORE\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "logreg = LogisticRegression()\n",
    "logreg.fit(X_train, y_train) \n",
    "y_pred = logreg.predict_proba(X_test_set)[:,1]\n",
    "roc_auc_score(y_test_set, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://medium.com/@faizanahemad/participating-in-kaggle-data-science-competitions-part-1-step-by-step-guide-and-baseline-model-5b0c6973022a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = application_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# How many classes\n",
    "df[\"TARGET\"].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Distribution of those classes\n",
    "df[\"TARGET\"].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtypes = df.dtypes\n",
    "dtypes = dtypes[dtypes != 'object']\n",
    "features = list(set(dtypes.index) - set(['TARGET']))\n",
    "len(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df[features]\n",
    "y = df['TARGET']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = XGBClassifier(max_depth=6,\n",
    "                      learning_rate=0.1,\n",
    "                      n_estimators=100,\n",
    "                      n_jobs=16,\n",
    "                      scale_pos_weight=4,\n",
    "                      missing=np.nan,\n",
    "                      gamma=16,\n",
    "                      eval_metric='auc',\n",
    "                      reg_lambda=40,reg_alpha=40\n",
    "                     )\n",
    "model.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 10: Scoring on Train and Test set\n",
    "\n",
    "We will predict probabilities of our TARGET=1,\n",
    "  P(TARGET=1|X) and use it for finding AUC_ROC metric."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "y_train_predicted = model.predict_proba(X_train)[:,1]\n",
    "y_test_predicted = model.predict_proba(X_test)[:,1]\n",
    "\n",
    "print('Train AUC %.4f' % roc_auc_score(y_train,y_train_predicted))\n",
    "print('Test AUC %.4f' % roc_auc_score(y_test,y_test_predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_results(model,df_test,features,id_col,target,file):\n",
    "    dft = df_test[features]\n",
    "    results = df_test[[id_col]]\n",
    "    results[target] = model.predict_proba(dft)[:,1]\n",
    "    results.to_csv(file,index=False,columns=results.columns\n",
    ")\n",
    "    \n",
    "generate_results(model,df_test,features,\"SK_ID_CURR\",\"TARGET\",\"results/results.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Jupyter Notebook - Typesetting Equations](http://jupyter-notebook.readthedocs.io/en/stable/examples/Notebook/Typesetting%20Equations.html)\n",
    "The Markdown parser included in the Jupyter Notebook is MathJax-aware. This means that you can freely mix in mathematical expressions using the [MathJax subset of Tex and LaTeX](http://docs.mathjax.org/en/latest/tex.html#tex-support). Some examples from the MathJax site are reproduced below, as well as the Markdown+TeX source.\n",
    "[MathJax basic tutorial and quick reference](https://math.meta.stackexchange.com/questions/5020/mathjax-basic-tutorial-and-quick-reference)\n",
    "\n",
    "\\begin{align}\n",
    "\\dot{x} & = \\sigma(y-x) \\\\\n",
    "\\dot{y} & = \\rho x - y - xz \\\\\n",
    "\\dot{z} & = -\\beta z + xy\n",
    "\\end{align}\n",
    "\n",
    "\\begin{equation*}\n",
    "P(E)   = {n \\choose k} p^k (1-p)^{ n-k}\n",
    "\\end{equation*}\n",
    "\n",
    "This expression $\\sqrt{3x-1}+(1+x)^2$ is an example of a TeX inline equation in a [Markdown-formatted](https://daringfireball.net/projects/markdown/) sentence.\n",
    "\n",
    "###### Other Syntax\n",
    "You will notice in other places on the web that $$ are needed explicitly to begin and end MathJax typesetting. This is not required if you will be using TeX environments, but the Jupyter notebook will accept this syntax on legacy notebooks.\n",
    "\n",
    "$$\\begin{eqnarray}\n",
    "x' &=& &x \\sin\\phi &+& z \\cos\\phi \\\\\n",
    "z' &=& - &x \\cos\\phi &+& z \\sin\\phi \\\\\n",
    "\\end{eqnarray}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{align}\n",
    "Y = \\beta_0 + \\beta_1 X_1 + \\beta_2 X_2 + \\beta_3 X_3 + \\beta_4 X_4\n",
    "\\end{align}\n",
    "\\begin{align}\n",
    "R^2\n",
    "\\end{align}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multicollinearity Detection\n",
    "https://statinfer.com/204-1-9-issue-of-multicollinearity-in-python/\n",
    "    Y = β0 + β1X1 + β2X2 + β3X3 + β4X4\n",
    "    Build a model X1 vs X2 X3 X4 find R^2, say R1\n",
    "    Build a model X2 vs X1 X3 X4 find R^2, say R2\n",
    "    Build a model X3 vs X1 X2 X4 find R^2, say R3\n",
    "    Build a model X4 vs X1 X2 X3 find R^2, say R4\n",
    "    For example if R3 is 95% then we don’t really need X3 in the model.\n",
    "    Since it can be explained as liner combination of other three.\n",
    "    For each variable we find individual R2.\n",
    "    1/(1 − R^2) is called VIF."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train__ext_source = application_train.loc[:, ['EXT_SOURCE_1','EXT_SOURCE_2','EXT_SOURCE_3']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove observations with missing values\n",
    "application_train__ext_source__dropna = application_train__ext_source.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "There is no function for calculating VIF values.\n",
    "None of the pre-built libraries have this VIF calculation function\n",
    "We may have to write our own function to calculate VIF values for each variable \n",
    "\"\"\"\n",
    "\n",
    "# Code for VIF Calculation\n",
    "# Writing a function to calculate the VIF values\n",
    "#def variable_inflation_factor_calculations(input_data, dependent_col):\n",
    "def vif_cal(input_data, dependent_col):\n",
    "    x_vars = input_data.drop([dependent_col], axis=1)\n",
    "    xvar_names = x_vars.columns\n",
    "    for i in range(0, xvar_names.shape[0]):\n",
    "        y = x_vars[xvar_names[i]] \n",
    "        x = x_vars[xvar_names.drop(xvar_names[i])]\n",
    "        rsq = smf.ols(formula='y ~ x', data = x_vars).fit().rsquared  \n",
    "        vif = round(1 / (1 - rsq), 2)\n",
    "        print (xvar_names[i], 'VIF = ' , vif)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# EXT_SOURCE_1 ~ EXT_SOURCE_2 + EXT_SOURCE_3\n",
    "linreg1 = LinearRegression()\n",
    "linreg1.fit(application_train__ext_source__dropna[['EXT_SOURCE_2'] + ['EXT_SOURCE_3']], application_train__ext_source__dropna[['EXT_SOURCE_1']])\n",
    "linreg1_predicted = linreg1.predict(application_train__ext_source__dropna[['EXT_SOURCE_2'] + ['EXT_SOURCE_3']])\n",
    "linreg1_model = smf.ols(formula='EXT_SOURCE_1 ~ EXT_SOURCE_2 + EXT_SOURCE_3', data=application_train__ext_source__dropna)\n",
    "linreg1_fitted = linreg1_model.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linreg1_fitted.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linreg1_fitted.summary2()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculating VIF values using that function\n",
    "vif_cal(input_data = application_train__ext_source__dropna, dependent_col = 'EXT_SOURCE_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# EXT_SOURCE_2 ~ EXT_SOURCE_3 + EXT_SOURCE_1\n",
    "linreg2 = LinearRegression()\n",
    "linreg2.fit(application_train__ext_source__dropna[['EXT_SOURCE_3'] + ['EXT_SOURCE_1']], application_train__ext_source__dropna[['EXT_SOURCE_2']])\n",
    "linreg2_predicted = linreg1.predict(application_train__ext_source__dropna[['EXT_SOURCE_3'] + ['EXT_SOURCE_1']])\n",
    "linreg2_model = smf.ols(formula='EXT_SOURCE_2 ~ EXT_SOURCE_3 + EXT_SOURCE_1', data=application_train__ext_source__dropna)\n",
    "linreg2_fitted = linreg1_model.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linreg2_fitted.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linreg2_fitted.summary2()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculating VIF values using that function\n",
    "vif_cal(input_data = application_train__ext_source__dropna, dependent_col = 'EXT_SOURCE_2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# EXT_SOURCE_3 ~ EXT_SOURCE_1 + EXT_SOURCE_2\n",
    "linreg3 = LinearRegression()\n",
    "linreg3.fit(application_train__ext_source__dropna[['EXT_SOURCE_1'] + ['EXT_SOURCE_2']], application_train__ext_source__dropna[['EXT_SOURCE_3']])\n",
    "linreg3_predicted = linreg1.predict(application_train__ext_source__dropna[['EXT_SOURCE_1'] + ['EXT_SOURCE_2']])\n",
    "linreg3_model = smf.ols(formula='EXT_SOURCE_3 ~ EXT_SOURCE_1 + EXT_SOURCE_2', data=application_train__ext_source__dropna)\n",
    "linreg3_fitted = linreg1_model.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linreg3_fitted.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linreg3_fitted.summary2()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculating VIF values using that function\n",
    "vif_cal(input_data = application_train__ext_source__dropna, dependent_col = 'EXT_SOURCE_3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/22470690/get-list-of-pandas-dataframe-columns-based-on-data-type\n",
    "# df.columns.to_series().groupby(df.dtypes).groups\n",
    "application_train.columns.to_series().groupby(application_train.dtypes).groups"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
